/**
 * 
 */
package com.spider.silence.downloader;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.net.URLDecoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

import com.spider.silence.utils.FileUtils;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.MaLaTask;
import com.spider.silence.task.MalaModuleTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.PropertiesUtil;

import us.codecraft.webmagic.Page;

/**
 * @title MaLaDownLoader.java
 * @author liyuanguo
 * @time 2016年12月28日 上午10:04:46
 * @description
 * @version V1.0
 */
public class MaLaDownLoader extends AbstractCommonDownLoader implements Cloneable {
	private Logger logger =Logger.getLogger(MaLaDownLoader.class);
	private String[] modules = { "" };

	private Map<String,File> files=null;

	public Map<String,File> getFiles(){

		return files;
	}

	public void generateTask() {
		Document doc = null;
		boolean flag = true;
		long startTime = System.currentTimeMillis();
		long nowTime = startTime;
		while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("MaLa.TimeOut"))) {
			try {
				// doc= get("http://www.mala.cn/forum.php",true);
				Page pageContent = null;
				pageContent = getHtml("http://www.mala.cn/forum.php", true);
				if (null == pageContent) {

				} else if (pageContent.getStatusCode() == 0) {

				} else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

				} else if (pageContent.getStatusCode() == 200) {
					flag = false;
					//System.out.println("爬取网站:" + "http://www.mala.cn/forum.php" + "状态码：" + pageContent.getStatusCode());
					doc = pageContent.getHtml().getDocument();
					Elements elementsData = doc.select("div[id^='category_']").select("dl");
					Elements elementss = new Elements();
					for (int i = 0; i < elementsData.size(); i++) {
                        try {
                            int hours = Integer.valueOf(PropertiesUtil.getVal("MaLa.ModuleTime"));
                            // 配置的起始时间
                            long dateStart = DateUtils.addHours(new Date(), hours).getTime();
                            // System.out.println("起始时间:"+dateStart);
                            String updateTime = "";
                            String updateTimeStr = elementsData.get(i).select("dd").get(1).select("a").text();
                            // System.out.println("最后发表时间内容："+updateTimeStr);
                            updateTime = updateTimeStr.substring(5, updateTimeStr.length()) + ":00";
                            // System.out.println("最后发表时间："+updateTime);
                            SimpleDateFormat localSimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                            long upTime = localSimpleDateFormat.parse(updateTime).getTime();
                            // System.out.println("最后发表时间Long:"+upTime);
                            if (upTime >= dateStart) {
                                Elements moduleUrls = elementsData.get(i).select("dt").select("a");
                                if (null != moduleUrls && moduleUrls.size() > 0) {
                                    elementss.add(moduleUrls.get(0));
                                }

                            }
                        } catch (NumberFormatException e) {
                        	logger.error(e.getMessage(),e);
                            e.printStackTrace();
                        } catch (Exception e) {
                        	logger.error(e.getMessage(),e);
                            continue;
                        }
                    }

					// 获取所有的模块
					// Elements
					// elementss=doc.select("div[id^='category_']").select("dt").select("a");
					for (int i = 0; i < elementss.size(); i++) {
						//System.out.println(elementss.get(i).attr("href"));
						MalaModuleTask task = new MalaModuleTask(elementss.get(i).attr("href"), elementss.size(), i);
						CommonTaskManager.add("malaModule", task);

					}

				} else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

				} else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

				} else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

				} else {

				}

			} catch (Exception e1) {
				//System.out.println("异常信息：" + e1.getMessage());
				logger.error(e1.getMessage(),e1);
			}finally{
				nowTime = System.currentTimeMillis();
			}
		}

	}

	public JSONObject crawlItem(String url) {
		files=new ConcurrentHashMap<>();
		JSONObject item = new JSONObject();
		JSONObject data = new JSONObject();
		int page = 1;
		//System.out.println("爬取的：" + url);
		try {
			// Document document=get(url,true);
			Document document = null;
			boolean flag = true;
			long startTime = System.currentTimeMillis();
			long nowTime = startTime;
			while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("MaLa.TimeOut"))) {
				try {
					// doc= get("http://www.mala.cn/forum.php",true);
					Page pageContent = null;
					pageContent = getHtml(url, true);
					if (null == pageContent) {

					} else if (pageContent.getStatusCode() == 0) {

					} else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

					} else if (pageContent.getStatusCode() == 200) {
						try {
							//System.out.println("爬取言论第一页:" + url + "状态码：" + pageContent.getStatusCode());
							document = pageContent.getHtml().getDocument();
							document.select(".aimg_tip").remove();
							String title = document.title();
							Elements pageInfos=document.select("#postlist");
							if(null!=pageInfos&&pageInfos.size()>0){
								Element pageInfo = pageInfos.get(0);
								String content = "";
								if (null != pageInfo.select(".t_f") && pageInfo.select(".t_f").size() > 0&&!"".equals(pageInfo.select(".t_f").get(0).html())) {

									// 爬取第一页上帖子的内容
									int read_count = Integer.parseInt(pageInfo.select("span.xi1").get(0).text());// 阅读量
									// System.out.println("阅读量："+read_count);
									int reply_count = Integer.parseInt(pageInfo.select("span.xi1").get(1).text());// 回复量
									// System.out.println("回复数："+reply_count);
									// page= (int)
									// Math.ceil(((double)reply_count+1)/20d);//页数
									Element pageI = document.select("#pgt").select(".pgt").get(0);
									if (null != pageI.html() && !"".equals(pageI.html())) {
										String pageString = pageI.select("span").text();
										page = Integer.valueOf(pageString.split(" ")[1]);
									}
									// System.out.println("页数："+page);
									String publish_time = "";// 发布时间
									publish_time = DateUtils.strToFormatStr(getTime(pageInfo.select(".authi").select("em").text())+ ":00");
									//System.out.println("发布时间："+publish_time);
									content = pageInfo.select(".t_f").get(0).text();// 发布内容
                                    Element eL=pageInfo.select(".t_f").get(0);
									JSONArray img_urls=new JSONArray();
									Elements imgElements=pageInfo.select(".t_f").get(0).select("img[class='zoom']");
									for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
										Element imgElement=imgElements.get(imgIndex);
										String img_url=imgElement.absUrl("zoomfile");
										if(img_url.isEmpty()){
                                            img_url=imgElement.absUrl("src");
                                        }
										img_urls.add(img_url);
										System.out.println(img_url);
									}

									//System.out.println("发布内容：" + content);
									String publish_user_id = getUrlParams(pageInfo.select(".authi").select("a").get(0).attr("href")).get("uid").toString();// 发布者的ID
									// System.out.println("用户id："+publish_user_id);
									String publish_user = pageInfo.select(".authi").select("a").get(0).text();// 发布用户
									//System.out.println("用户:"+publish_user);
									String publish_user_photo = pageInfo.select(".avatar").select("img").get(0).attr("src");
									//System.out.println("用户头像："+publish_user_photo);
									String[] urlParam = url.split("-");
									String id = urlParam[1];// 帖子的Id
									// System.out.println("帖子id："+id);
									try {
										String like = pageInfo.select("#recommendv_add").text();
										// System.out.println("点赞数："+like);
										String dislike = pageInfo.select("#recommendv_subtract").text();
										// System.out.println("反对数："+dislike);
										if (!"".equals(like) && like != null) {
											int like_count = Integer.parseInt(like);
											data.put("like_count", like_count);
										} else {
											data.put("like_count", 0);
										}
										if (!"".equals(dislike) && dislike != null) {
											int dislike_count = Integer.parseInt(dislike);
											data.put("dislike_count", dislike_count);
										} else {
											data.put("dislike_count", 0);
										}
									} catch (Exception e) {
										logger.error(e.getMessage(),e);
									}

									data.put("reply_count", reply_count);
									data.put("read_count", read_count);
									data.put("title", (title));
									data.put("url", url);
									data.put("publish_time", publish_time);
									Date now = new Date();
									SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
									data.put("spider_time", sdf.format(now));
									data.put("reproduce_count", 0);
									data.put("content", (content));
									data.put("publish_user_id", publish_user_id);
									data.put("publish_user", (publish_user));
									data.put("publish_user_photo", publish_user_photo);
									data.put("id", id);
                                    data.put("img_urls",img_urls);
									long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
									String filePrefix="MaLa";//文件前缀
									String fileName=filePrefix+"_"+tsp+"_"+publish_user_id+"_"+id;
									String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
									String speechesId="MaLa_speeches_"+publishDate+"_"+fileName;
									File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
									files.put(speechesId+"_pageNo_1",file);
									//帖子首页的文件
									Elements replyEntities = document.select("#postlist").select("div[id^='post_']:gt(2)");
									JSONArray reply_nodes = new JSONArray();
									// System.out.println("回复："+replyEntities);

									// 爬取第一页的回复
									for (int i = 0; i < replyEntities.size(); i++) {
										try {
											Element replyUserInfo = replyEntities.get(i).select("td.pls").get(0);
											//System.out.println("回复用户信息："+replyUserInfo);
											JSONObject reply = new JSONObject();
											String replyUser = "";
											String replyUserId = "";
											Elements es = replyUserInfo.select(".authi").select("a");
											if (null != replyUserInfo.select(".authi").select("a")
													&& replyUserInfo.select(".authi").select("a").size() > 0) {
												replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
												replyUserId=getUrlParams(replyUserInfo.select(".authi").select("a").get(0).attr("href")).get("uid").toString();
											} else if (null != replyUserInfo.select("em")
													&& replyUserInfo.select("em").size() > 0) {
												replyUser = replyUserInfo.select("em").get(0).text();
											}
											//System.out.println("回复用户名："+replyUser);
											// String
											// homeUrl=replyUserInfo.select(".authi").select("a").get(0).attr("href");
											String homeUrl = "";
											// String
											// replyUserId=getUrlParams(homeUrl).get("uid").toString();
											// System.out.println("头像内容："+replyUserInfo.select(".avatar"));
											String replyUserPhoto = replyUserInfo.select(".avatar").select("img").get(0)
													.attr("src");
											//System.out.println("回复者头像："+replyUserPhoto);
											String replyContent = replyEntities.get(i).select(".plc").select(".pct").get(0).text();
											JSONArray reply_img_urls=new JSONArray();
											Elements replyImgElements=replyEntities.get(i).select(".plc").select(".pct").get(0).select("img[class='zoom']");
											for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
												Element imgElement=replyImgElements.get(imgIndex);
												String img_url=imgElement.absUrl("zoomfile");
                                                if(img_url.isEmpty()){
                                                    img_url=imgElement.absUrl("src");
                                                }
												reply_img_urls.add(img_url);
                                            }
											//System.out.println("回复内容："+replyContent);
											String replyTime = DateUtils.strToFormatStr(getTime(replyEntities.get(i).select(".plc").select(".authi").select("em").html())+ ":00");
											//System.out.println("回复时间："+replyTime);
											String replyUrl = url + "#" + replyEntities.get(i).id();
											// System.out.println("回复url："+replyUrl);

											reply.put("publish_user", replyUser);
											reply.put("publish_user_id", replyUserId);
											reply.put("publish_user_photo", replyUserPhoto);
											reply.put("content", replyContent);
											reply.put("publish_time", replyTime);
											reply.put("url", replyUrl);
											reply.put("parent_id", id);
											reply.put("ancestor_id", id);
											reply.put("img_urls",reply_img_urls);
											reply_nodes.add(reply);
										} catch (Exception e) {
											logger.error("因为禁言无法获取的情况",e);
											continue;// 因为禁言无法获取的情况
										}

									}

									JSONArray replys = new JSONArray();
									List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();

									// 爬取2页之后每页的回复
									if (page >= 2) {
										int threadNum = Integer.valueOf(PropertiesUtil.getVal("MaLa.ReplyThreadCount"));
										List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
										// 线程池 初始化十个线程 和JDBC连接池是一个意思 实现重用
										ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
										try {
											ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
											for (int i = 0; i < splitPages.size(); i++) {
												MaLaDownLoader downLoader = this.cloneDownLoader();
												ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), url, id,speechesId);
												FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
												futureTasks.add(futureTask);
												executorService.submit(futureTask);
											}
											int count = 0;

											for (FutureTask<JSONArray> futureTask : futureTasks) {
												JSONArray replyArrays = futureTask.get();
												for (int i = 0; i < replyArrays.size(); i++) {
													JSONObject object = replyArrays.getJSONObject(i);
													replys.add(object);
												}
											}
											if (replys.size() > 0 && replys != null) {
												for (int i = 0; i < replys.size(); i++) {
													JSONObject reply_object = replys.getJSONObject(i);
													reply_nodes.add(reply_object);
												}
											}
											if (reply_nodes.size() > 0 && reply_nodes != null) {
												for (int i = 0; i < reply_nodes.size(); i++) {
													JSONObject reply_object = reply_nodes.getJSONObject(i);
													Map<String, Object> map = reply_object;
													list.add(map);
												}
											}
											Collections.sort(list, new Comparator<Map<String, Object>>() {

												public int compare(Map<String, Object> o1, Map<String, Object> o2) {
													String s1 = o1.get("publish_time").toString();
													String s2 = o2.get("publish_time").toString();
													int flag = s1.compareTo(s2);
													return flag;
												}
											});
										}catch (Exception ee) {
											logger.error(ee.getMessage(),ee);
											ee.printStackTrace();
										} finally {
											executorService.shutdown();
										}
									} else {
										if (reply_nodes.size() > 0 && reply_nodes != null) {
											for (int i = 0; i < reply_nodes.size(); i++) {
												JSONObject reply_object = reply_nodes.getJSONObject(i);
												Map<String, Object> map = reply_object;
												list.add(map);
											}
										}
									}
									data.put("reply_nodes", list);
								} else {

								}
							}else{

							}

						} catch (Exception e) {

							logger.error(e.getMessage(),e);
						}finally{
							flag = false;
						}

					} else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

					} else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

					} else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

					} else {

					}

				} catch (Exception e1) {
					//System.out.println("异常信息：" + e1.getMessage());
					logger.error(e1.getMessage(),e1);
				}finally{
					if(flag==true){

						item = new JSONObject();
						data = new JSONObject();
					}
					nowTime = System.currentTimeMillis();
				}
			}

		} catch (Exception e) {
			item = new JSONObject();
			data = new JSONObject();
			logger.error(e.getMessage(),e);
		}
		item.put("data", data);
		return item;
	}

	public MaLaDownLoader cloneDownLoader()
			throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
		MaLaDownLoader loader = new MaLaDownLoader();
		loader.cookies = (HashSet<Cookie>) this.cookies.clone();
		loader.headers = (HashMap<String, String>) this.headers.clone();
		return loader;
	}

	public class ReplyCrawler implements Callable<JSONArray> {
		private MaLaDownLoader downLoader;
		private ArrayList<Integer> pages;
		private String url;
		private String parent_id;
		private String speeches_id;

		@Override
		public JSONArray call() throws Exception {
			JSONArray array = new JSONArray();
			for (int k = 0; k < pages.size(); k++) {
				JSONArray arrayPage = new JSONArray();
				try {

					boolean flag = true;
					long startTime = System.currentTimeMillis();
					long nowTime = startTime;
					while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("MaLa.TimeOut"))) {
						try {
							// doc= get("http://www.mala.cn/forum.php",true);
							String[] urlParam = url.split("-");
							// System.out.println("页码："+pages.get(k));
							String getUrl = urlParam[0] + "-" + urlParam[1] + "-" + pages.get(k) + "-" + urlParam[3];
							Document document = null;
							Page pageContent = null;
							pageContent = downLoader.getHtml(getUrl, true);
							if (null == pageContent) {

							} else if (pageContent.getStatusCode() == 0) {

							} else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

							} else if (pageContent.getStatusCode() == 200) {
								//System.out.println("爬取回复:" + getUrl + "状态码：" + pageContent.getStatusCode());
								document = pageContent.getHtml().getDocument();
								document.select(".aimg_tip").remove();
								File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
								files.put(speeches_id+"_pageNo_"+pages.get(k),file);
								Elements replyEntities = document.select("#postlist").select("div[id^='post_']:gt(1)");
								JSONArray reply_nodes = new JSONArray();
								for (int i = 0; i < replyEntities.size(); i++) {
									try {
										Element replyUserInfo = replyEntities.get(i).select("td.pls").get(0);
										// System.out.println("回复用户信息："+replyUserInfo);
										JSONObject reply = new JSONObject();
										String replyUser = "";
										String replyUserId = "";
										if (null != replyUserInfo.select(".authi").select("a")
												&& replyUserInfo.select(".authi").select("a").size() > 0) {
											replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
											replyUserId=getUrlParams(replyUserInfo.select(".authi").select("a").get(0).attr("href")).get("uid").toString();
										} else if (null != replyUserInfo.select("em")
												&& replyUserInfo.select("em").size() > 0) {
											replyUser = replyUserInfo.select("em").get(0).text();
										}
										//System.out.println("回复用户名："+replyUser);
										// String
										// homeUrl=replyUserInfo.select(".authi").select("a").get(0).attr("href");
										String homeUrl = "";
										// String
										// replyUserId=getUrlParams(homeUrl).get("uid").toString();
										//System.out.println("头像内容："+replyUserInfo.select(".avatar"));
										String replyUserPhoto = replyUserInfo.select(".avatar").select("img").get(0)
												.attr("src");
										//System.out.println("回复者头像："+replyUserPhoto);
										//String replyContent = replyEntities.get(i).select(".t_f").html();
										String replyContent = replyEntities.get(i).select(".plc").select(".pct").get(0).text();
										JSONArray reply_img_urls=new JSONArray();
										Elements reply_imgElements=replyEntities.get(i).select(".plc").select(".pct").get(0).select("img[class='zoom']");
										for(int imgIndex=0;imgIndex<reply_imgElements.size();imgIndex++){
										    Element imgElemnt=reply_imgElements.get(imgIndex);
										    String img_url=imgElemnt.absUrl("zoomfile");
										    if(img_url.isEmpty()){
                                                img_url=imgElemnt.absUrl("src");
                                            }
                                            reply_img_urls.add(img_url);
                                            System.out.println(img_url);
                                        }

										//System.out.println("回复内容："+replyContent);
										String replyTime = DateUtils.strToFormatStr(getTime(replyEntities.get(i).select(".plc").select(".authi").select("em").html()) + ":00");
										//System.out.println("回复时间："+replyTime);
										String replyUrl = url + "#" + replyEntities.get(i).id();
										// System.out.println("回复url："+replyUrl);

										reply.put("publish_user", (replyUser));
										reply.put("publish_user_id", replyUserId);
										reply.put("publish_user_photo", replyUserPhoto);
										reply.put("content", (replyContent));
										reply.put("publish_time", replyTime);
										reply.put("url", replyUrl);
										reply.put("parent_id", parent_id);
										reply.put("ancestor_id", parent_id);
                                        reply.put("img_urls", reply_img_urls);
										arrayPage.add(reply);

									} catch (Exception e) {
										logger.error("因为禁言无法获取的情况",e);
										continue;// 因为禁言无法获取的情况
									}
								}
								flag = false;
							} else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

							} else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

							} else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

							} else {

							}

						} catch (Exception e1) {
							//System.out.println("异常信息：" + e1.getMessage());

							logger.error(e1.getMessage());
						}finally{
							if(flag==false){
								array.addAll(arrayPage);
							}
							nowTime = System.currentTimeMillis();
						}
					}

				} catch (Exception e) {
					logger.error(e.getMessage(),e);
					continue;
				}
			}
			return array;
		}

		public ReplyCrawler(MaLaDownLoader downLoader, ArrayList<Integer> pages, String url, String parent_id,String speeches_id) {
			this.downLoader = downLoader;
			this.pages = pages;
			this.url = url;
			this.parent_id = parent_id;
			this.speeches_id=speeches_id;
		}
	}
}
